package com.darkflame.client.semantic;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.logging.Logger;

import com.darkflame.client.SuperSimpleSemantics;
import com.darkflame.client.interfaces.GenericDebugDisplayer;
import com.darkflame.client.query.Query;
import com.darkflame.client.query.QueryElement;
import com.darkflame.client.query.QuerySSSProperty;
import com.darkflame.client.query.Query.QueryMode;



/** This class runs supplied query and returns the results.
 * In order to do this, we have to trigger a fairly complex set of asycronominious tasks. **/
public class QueryEngine {
	static Logger Log = Logger.getLogger("sss.QueryEngine");

	/** Main cache of queries.<br>
	 * This probably should be capped at some reasonable number<br>
	 * but I don't know what that number is...hmm<br>
	 * We use a LinkedHashMap, as thats like a hashmap but maintains an order.<br>
	 * Without an order we don't know what's the "oldest" element to remove when it gets too big*/
	private static final LinkedHashMap<Query,ArrayList<SSSNode>> QueryCache = new LinkedHashMap<Query,ArrayList<SSSNode>>();

	/** The number of query results to keep cached **/
	private static final int QueryCacheLimit = 30;

	/** the index's should all be loaded before calling this.<br>
	 * This function will retrieve all the SSSFiles that have a reliance to this query<br>
	 * then it will get the matching nodes from them. <br> **/
	static public ArrayList<String> getAllSubclassURLs(SSSNode value){

		// first we need to get all possible subclass's of the value
		// so we look for all "subclass's"
		// for example types of green
		ArrayList<String> subclasss = SSSIndex.getAllSSSFileURLsFor(SSSNode.SubClassOf, value);
		Iterator<String> subClasssIt = subclasss.iterator();

		while (subClasssIt.hasNext()) {

			final String string = (String) subClasssIt.next();

			//set it loading, and on completion set it to run this function again, till all are loaded
			SuperSimpleSemantics.info("urls to load for subclass:"+string);

			//SSSNodesWithCommonProperty subclass = new SSSNodesWithCommonProperty(SSSNode.SubClassOf, value);
			SSSNodesWithCommonProperty subclass = SSSNodesWithCommonProperty.createSSSNodesWithCommonProperty(SSSNode.SubClassOf, value);

			//???
			SuperSimpleSemantics.info("file load triggering from \"getAllSubclass\" - might be a mistake in code sorry");
			subclass.loadSSSFile(string);

			subclass.setCallback(new LoadingCallback() {

				@Override
				public void onSuccess(String result) {
					Log.info("url loaded for subclass:"+string);
					//now load all results too...

				}

				@Override
				public void onFailure(Throwable caught) {

				}
			});

			//once its loaded, check the new class's

		}
		//		 
		//		 
		//		 
		//		//get relevant list urls
		//		//these arnt just direct matches, we also need everything that is a classOf the value as well, and those become addition values to search for
		//		
		//		 ArrayList<String> PropFiles = SSSIndex.getAllSSSFileURLsFor(pred, value);
		//		
		//		 Iterator<String> sssfiles = PropFiles.iterator();
		//	
		//		 while (sssfiles.hasNext()) {
		//		String sssfileurl = (String) sssfiles.next();
		//		
		//		
		//		PropFiles.add(sssfileurl);
		return subclasss;

		//}
		//ensure all sssfiles are freshly loaded (can put a cache here latter)
	}

	/** gets all the nodes which are a subclass (rdfs:subClassOf) of the specified value **/
	static public void getNodesWhichAre(SSSNode val,DoSomethingWithNodesRunnable dothisafter){


		SuperSimpleSemantics.setSearchClockTotal(1);
		SuperSimpleSemantics.setSearchClockProgress(0);	

		SSSNodesWithCommonProperty.getAllNodesWithProperty(SSSNode.SubClassOf, val,dothisafter,false);

		return;
	}
	

	/** Gets the currently known nodes which are a subclass of the specified value <br>
	 * This is only safe to use if all SSSnodes are already loaded **/ 
	static public HashSet<SSSNode> getCurrentNodesWhichAre(SSSNode val){			


		SuperSimpleSemantics.setSearchClockTotal(1);
		SuperSimpleSemantics.setSearchClockProgress(0);	

		return SSSNodesWithCommonProperty.getAllCurrentNodesWithProperty(SSSNode.SubClassOf, val,false);

		//		
	}
	/** Get nodes with this specified property.<br>
	 * If you wish to see what URIs support this conclusion use<br>
	 * <br>
	 * SSSNodesWithCommonPropety.getURLsThatSupportThisTriplet()<br>
	 * In future there will be a helper method for this. **/

	static public HashSet<SSSNode> getCurrentNodesWithProperty(SSSNode pred, SSSNode val){

		SuperSimpleSemantics.setSearchClockTotal(1);
		SuperSimpleSemantics.setSearchClockProgress(0);

		return SSSNodesWithCommonProperty.getAllCurrentNodesWithProperty(pred, val,false);
		//		
	}

	/** get nodes with this specified property.<br>
	 * If you wish to see what URIs support this conclusion use<br>
	 * <br>
	 * SSSNodesWithCommonPropety.getURLsThatSupportThisTriplet()<br>
	 * In future there will be a helper method for this. **/

	static public void getNodesWithProperty(SSSNode pred, SSSNode val,DoSomethingWithNodesRunnable dothisafter){

		SuperSimpleSemantics.setSearchClockTotal(1);
		SuperSimpleSemantics.setSearchClockProgress(0);
		SSSNodesWithCommonProperty.getAllNodesWithProperty(pred, val,dothisafter,false);

		return; 
		//		
	}
	
	
	
	/**
	 * Old method to output a string for querying.
	 * use Query objects instead.
	 * 
	 * @param usersEntry
	 * @return
	 */
	@Deprecated
	public static String processRawQuery_depreciated(String usersEntry) {


		Log.info("interpreting user words");

		//break to words
		String words[] = usersEntry.split(" "); 
		String builtQuery = "";
		boolean quoteOpen=false;
		String currentWord ="";
		//loop  
		for (String word : words) {

			currentWord= (currentWord+" "+word).trim();


			//if its quoted we get the whole word
			if (currentWord.startsWith("\"")&&!quoteOpen){
				if (!currentWord.substring(1).contains("\"")){
					quoteOpen=true;
				}				
			} else if (currentWord.substring(1).contains("\"")&& quoteOpen){
				//quote ended
				quoteOpen=false;
				//remove quotes
				currentWord = currentWord.replace("\"", "");

				Log.info("currentWord="+currentWord);

			}

			if (quoteOpen){
				continue;
			}

			word=currentWord;
			currentWord="";

			//if its a property then split
			if (word.contains("=")){

				String pred = word.split("=")[0];
				String val = word.split("=")[1];

				if (!pred.contains(":")&&(!pred.contains("."))){

					pred = SSSNode.getNodeByLabel(pred).PURI;
				}
				if (!val.contains(":")&&(!val.contains("."))){

					val = SSSNode.getNodeByLabel(val).PURI;
				}

				word = pred+"<~"+val;

			} else {

				//if it does not contain : or "." assume its a label
				if (!word.contains(":")&&(!word.contains("."))){

					SSSNode requestednode = SSSNode.getNodeByLabel(word);
					if (requestednode!=null){
						word = requestednode.PURI;
					}
				}

			}

			if (word!=null){

				builtQuery = builtQuery+" "+word;

			} else {

				builtQuery = builtQuery+" [unrecognised]";

			}


		}



		//guess uris based on labels

		return builtQuery;
	}
	//
	//	public static ArrayList<SSSNode> processQuery_depreciated(String realQuery) {
	//
	//
	//
	//		//break to words
	//		String words[] = realQuery.split(" "); //must support quotes in future with proper parser
	//		String builtQuery = "";
	//
	//		ArrayList<SSSProperty> hasToHave = new ArrayList<SSSProperty>();
	//		//loop to get nodes
	//		for (String word : words) {
	//
	//			if (word.length()>2){
	//
	//				//if its specifying a class
	//				if (word.contains("<~")){
	//
	//					String pred = word.split("<~")[0];
	//					String val = word.split("<~")[1];
	//
	//					Log.info("getting pred:"+pred);
	//
	//					SSSNode predNode = SSSNode.getNodeByUri(pred);
	//
	//					Log.info("getting val:"+val);
	//
	//					SSSNode valNode = SSSNode.getNodeByUri(val);
	//
	//					if ((predNode==null)||(predNode==SSSNode.NOTFOUND)){
	//						Log.info("no matching nodes found for pred: "+pred);						
	//						return null;
	//					}
	//
	//					if ((valNode==null)||(valNode==SSSNode.NOTFOUND)){
	//						Log.info("no matching nodes found for val: "+val);						
	//						return null;
	//					}
	//
	//					hasToHave.add(new SSSProperty(predNode,valNode));
	//
	//				}  else {
	//
	//					Log.info("getting node:"+word);
	//
	//					SSSNode node = SSSNode.getNodeByUri(word);
	//
	//					if (node==null||(node==SSSNode.NOTFOUND)){
	//						Log.info("no matching node found for "+word);
	//						return null;
	//					}
	//
	//					hasToHave.add(new SSSProperty(SSSNode.SubClassOf,node));
	//
	//				}
	//			}
	//		}
	//
	//		ArrayList<SSSNode> results = getInsectionOfPropertys(hasToHave);
	//
	//		return results;
	//	}



	//will be used in new instant results mode of some query functions
	/**
	 * Stores a ArrayList of SSSNodes and a boolean to determine if they should be inverted or not.
	 * (That is, if they represent a set to include or exclude from another set of nodes)
	 * 
	 * @author Tom
	 *
	 */
	public static class NodesAndInvertStatus {
		ArrayList<SSSNode> nodes;		
		boolean invert = false;

		public NodesAndInvertStatus(ArrayList<SSSNode> nodes, boolean invert) {
			this.nodes = nodes;
			this.invert = invert;
		}
	}


	public static interface DoSomethingWithNodesRunnable {		

		void run(ArrayList<SSSNode> newnodes, boolean invert);


	}
	
	
	
	/**
	 * Returns the known results from the query at this moment, without doing more loading or asycn actions.<br>
	 * Use this only when needed! The normal processQuery function will be more accurate, allowing fractualar dynamic loading where needed.<br>
	 * <br>
	 * NOTE: this has been tested to work with basic queries, but its likely some more advanced stuff - like subqueries for preds or values dont work.
	 * 
	 * @param realQuery
	 * @param invert
	 * @return
	 */
	public static ArrayList<SSSNode> processQueryNOW(Query realQuery,Boolean invert) {
		
		//First we check the query cache for any existing queries that match this
		if (QueryCache.containsKey(realQuery)){

			Log.info("found result in cache!");
			ArrayList<SSSNode> cachedResult = QueryCache.get(realQuery);
					
			//no need to do more!
			return cachedResult;
		}
		
				
		ArrayList<SSSNode> results = null;
		if (realQuery.querymode == QueryMode.AND){
			Log.info("processing AND query object right now:");
			//get intersection style query
			results=getInsectionOfQueryNOW(realQuery,invert);
		}

		if (realQuery.querymode == QueryMode.OR){
			Log.info("processing OR query object right now:");
			//get combo style query
			results=getCombinationStyleQueryNOW(realQuery,invert);

		}
		
		//This function tidys the result up a bit and stores it in the cache
		ArrayList<SSSNode> polishedresult = cleanResultAndStoreInCache(
				realQuery, results);
		
		return polishedresult;

	}


	/** processes a OR query instantly without loading more stuff
	 * 
	 * @param runWhenDone 
	 * @param runOnEachNewResult **/
	private static ArrayList<SSSNode> getCombinationStyleQueryNOW(Query anyofthese,boolean invertResults) {
		
		
		Log.info("processing from OR query object:");
		boolean invertFlag = false;
		
		//get intersection
		ArrayList<SSSNode> results = new ArrayList<SSSNode>();
		//loop to get nodes
		for (QueryElement queryelement : anyofthese) {
			invertFlag = false;
			HashSet<SSSNode> node;
			//if the element is a query we regress
			if (queryelement.query!=null){
				
				Log.info("processing from  a sub query object::");
				Log.info("sub query =:"+queryelement.query.toString());
				
				ArrayList<SSSNode> subresults =  processQueryNOW(queryelement.query,invertResults);
				
				node = new HashSet<SSSNode>(subresults);				
				invertFlag = queryelement.query.invertResults;
				Log.info("invert results ="+invertFlag);
				
			} else {
			
				SSSProperty cvalue = queryelement.prop;
				//this whole sections needs to have callbacks to allow asycronise updates
				node = SSSNodesWithCommonProperty.getAllCurrentNodesWithProperty(cvalue.pred, cvalue.value,invertResults);
			
			}
			
			
			//note: we should support NOT here I think - if a subquery was a negative type 
			//(either Nand or Nor) then we subtract rather then add the nodes?
			//I think :p
			if (invertFlag)	{
				Log.info("removing nodes:"+node.size());
				results.removeAll(node);
			} else {
				Log.info("adding nodes:"+node.size());
				results.addAll(node);
			}
				
		}
		
		//runWhenDone.run(results);
		return results;
		
	}

	/** get nodes which have all these properties  
	 * @param runWhenDone 
	 * @param runOnEachNewResult **/
	public static ArrayList<SSSNode>  getInsectionOfQueryNOW(Query hasToHave,Boolean InvertResults) {
		
		Log.info("processing from query object:");
		
		boolean firstresult = true;
		boolean invertFlag = false;
		
		//get intersection
		ArrayList<SSSNode> results = new ArrayList<SSSNode>();
		//loop to get nodes
		for (QueryElement queryelement : hasToHave) {
			
			HashSet<SSSNode> node;
			//if the element is a query we regress
			if (queryelement.query!=null){
				
				Log.info("processing from  a sub query object:");
				Log.info("sub query =:"+queryelement.query.toString());
				
				ArrayList<SSSNode> subresults =  processQueryNOW(queryelement.query,InvertResults);
				
				node = new HashSet<SSSNode>(subresults);
				invertFlag = queryelement.query.invertResults;
				Log.info("invert results ="+invertFlag);
				
			} else {
			
				SSSProperty cvalue = queryelement.prop;
				node = SSSNodesWithCommonProperty.getAllCurrentNodesWithProperty(cvalue.pred, cvalue.value,InvertResults);
			
			}
	
			//the following should all be in a callback triggered by both above methods....
			
			Log.info("getting nodes:"+node.size());
			
			
			if (results.size()==0){
				
				results.addAll(node);
				firstresult = false;
				
			} else {
				
				if (invertFlag)	{
					Log.info("removing nodes:"+node.size());
					results.removeAll(node);
				} else {
					Log.info("retaining nodes:"+node.size());
					results.retainAll(node);
				}
								
			}
		}
		
	//	runWhenDone.run(results);
		return results;
	}


	
	/**
	 * Use this function to run a query and retrieve its results into a RunWhenDone.
	 * This is intended to be used as the primarily way to retrieve nodes that match your criteria.
	 * It will automatically load external files if needed, using "fractualar loading" till all the data is retrieved. 	 
	 * 
	 * If a a identical query is found in the cache, the cached result is returned instead as this is a lot faster.
	 * 
	 * NOTE: If there has been changes to the nodes between searches the cache might be out of date and should be cleared before searching.
	 * You can do this with the .clearQueryCache() command
	 * 
	 * @param realQuery - Query object that specifies the requirements to be searched for 
	 * @param invert - normally set to false, this is only used if part of a subquery and the results are to be subtracted from the outer one.
	 * @param RunOnEachNewResult - lets a UI reflect changes to currently retrieved results (experimental)
	 * @param RunWhenDone - final results get put here
	 */
	public static void processQuery(final Query realQuery,Boolean invert, DoSomethingWithNodesRunnable RunOnEachNewResult, final DoSomethingWithNodesRunnable RunWhenDone) {

		//First we check the query cache for any existing querys that match this
		if (QueryCache.containsKey(realQuery)){

			Log.info("found result in cache!");

			ArrayList<SSSNode> cachedResult = QueryCache.get(realQuery);
			
			RunWhenDone.run(cachedResult, invert);
			
			//no need to do more!
			return;
		}
		
		DoSomethingWithNodesRunnable cleanResultRunable = null;
		
		//we create a new run when done that cleans up the results before passing
		//to the user specified run when done.
		 cleanResultRunable = new DoSomethingWithNodesRunnable(){

			@Override
			public void run(ArrayList<SSSNode> newnodes, boolean invert) {

				Log.info("returning results after cleaning"+invert);

				ArrayList<SSSNode> polishedresult = cleanResultAndStoreInCache(
						realQuery, newnodes);
				//trigger the originally requested runnable with the new clean results
				//REMEMBER: this is not necessarily the last thing in the query
				//A subquery will fire this clean command before proceeding to the outer queries
				//Its thus important to pass on the invert flag still!
				RunWhenDone.run(polishedresult, invert);

			}

		};
		
		
		
		if (realQuery.querymode == QueryMode.AND){
			Log.info("processing AND query object:");
			//get intersection
			//results= 
			 getInsectionOfQuery(realQuery, RunOnEachNewResult, cleanResultRunable,invert);
		}

		if (realQuery.querymode == QueryMode.OR){
			Log.info("processing OR query object:");
			//get combo
			 getCombinationStyleQueryNewVersion(realQuery, RunOnEachNewResult, cleanResultRunable,invert);

		}



		return;


	}




	/** processes a OR query 
	 * @param runWhenDone 
	 * @param runOnEachNewResult 
	 * 
	 * @return - expiremental instant best guess. Dont use unless you have to. Use runWhenDone which is accurate.**/
	private static ArrayList<SSSNode>  getCombinationStyleQueryNewVersion(final Query anyofthese, DoSomethingWithNodesRunnable runOnEachNewResult, final DoSomethingWithNodesRunnable runWhenDone,final boolean invert) {


		log(" -------------------------------------- ");	

		log("Starting Combo query of - "+anyofthese.getAsString());
		log(" ------------------------------ ");

		Log.info("processing from OR query object:");
		boolean invertFlag = false;

		//establish if we are using a callback system to get the results. If we are, then InstantMode is set to false.	
		//This is the default, and the most powerful method. It allows dynamic cross-site loading of lists and sublists. 
		//if InstantMode is on, the method will instead just return currently known nodes that match all parameters. No extra loading will take place.
		boolean InstantModetemp = false;		
		if (runWhenDone==null){
			InstantModetemp=true;
		}
		final boolean InstantMode = InstantModetemp;		


		//processNewComboResult dothisafter = new processNewComboResult(runWhenDone);

		final ArrayList<SSSNode> results = new ArrayList<SSSNode>();
		final int LeftToLoadTotal = anyofthese.size();

		//first we declare a new callback
		//this will process the results after they come in and add them to the results list
		final DoSomethingWithNodesRunnable  dothisafter = new DoSomethingWithNodesRunnable (){	
			Query SourceQuery = anyofthese;
			private boolean invertFlag=false;
			int LeftToLoad =LeftToLoadTotal;
			@Override
			public void run(ArrayList<SSSNode> newnodes, boolean invert) {
				this.invertFlag = invert;
				//we do a hashset to remove dups
				HashSet<SSSNode> resultNodes = new HashSet<SSSNode>(newnodes);			


				Log.info("invert results ="+SourceQuery.invertResults);
				log("current query == "+SourceQuery.getAsString());
				log("invert results = "+invert);
				log("current results pre combine - "+results.toString());
				log("new nodes to combine:"+resultNodes.toString());

				//	if (results.size()==0){
				//	log("adding nodes as its first:"+resultNodes.size());
				//	results.addAll(resultNodes);

				//} else {
				if (invert)	{
					log("removing nodes:"+resultNodes.size());
					results.removeAll(resultNodes);
				} else {
					log("adding nodes:"+resultNodes.size());
					results.addAll(resultNodes);
				}

				//}

				log("current results after combine:"+results.toString());

				LeftToLoad--;
				log("current left to process = "+LeftToLoad);


				if (LeftToLoad==0){

					SourceQuery.setResultData(results.toString());

					log("triggering next step _"+SourceQuery.invertResults);

					runWhenDone.run(results,SourceQuery.invertResults);

				}
			}

		};


		//get intersection
		//ArrayList<SSSNode> results = new ArrayList<SSSNode>();
		//processNewComboResult.LeftToLoad = anyofthese.size();

		//loop to get nodes
		for (QueryElement queryelement : anyofthese) {
			//	invertFlag = false;
			HashSet<SSSNode> node;
			//if the element is a query we regress
			if (queryelement.query!=null){

				Log.info("processing from  a sub query object::");
				Log.info("sub query =:"+queryelement.query.getAsString());

				//invertFlag = queryelement.query.invertResults;
				Log.info("invert results ="+invertFlag);

				final QueryElement forwardquery = queryelement;		


				SuperSimpleSemantics.waitFor.scheduleAfter(new Runnable() {					
					@Override
					public void run() {
						processQuery(forwardquery.query,forwardquery.query.invertResults,null,dothisafter);

					}
				});

				/*
				Scheduler.get().scheduleDeferred(new ScheduledCommand() {    
					@Override
					public void execute() {
						processQuery(forwardquery.query,forwardquery.query.invertResults,null,dothisafter);
					}});
				 */

				//not in the above yet
				//processNewResult dothisafter = new processNewResult(runWhenDone);
				//	dothisafter.run(subresults,invertFlag);

				//node = new HashSet<SSSNode>(subresults);				


			} else {

				final QuerySSSProperty cvalue = queryelement.prop;

				final Boolean sinvertFlag = invertFlag;

				//the following should wait for any subquerys necessary first
				//HashSet<SSSNode> node = 
				if (InstantMode){
					if (cvalue.getQueryForPred()!=null){
						error("SubqueryPredicate not yet supported ");

					}
					if (cvalue.getQueryForValue()!=null){
						error("SubqueryValue not yet supported ");
					}
					
					
					HashSet<SSSNode> currentNodesWithPropertyHS = SSSNodesWithCommonProperty.getAllCurrentNodesWithProperty(cvalue.pred, cvalue.value,false);

					//convert to arraylist (fudge, we should convert everything to hashsets eventually?)
					ArrayList<SSSNode> currentNodesWithProperty = new ArrayList<SSSNode>(currentNodesWithPropertyHS);
					dothisafter.run(currentNodesWithProperty, false);
					
					
				}  else {
				
				SuperSimpleSemantics.waitFor.scheduleAfter(new Runnable() {

					@Override
					public void run() {

						if (cvalue.getQueryForPred()!=null){
							error("SubqueryPredicate not yet supported ");

						}
						if (cvalue.getQueryForValue()!=null){
							error("SubqueryValue not yet supported ");
						}


						DoSomethingWithNodesRunnable doThisAfter = new DoSomethingWithNodesRunnable(){

							@Override
							public void run(ArrayList<SSSNode> newnodes,
									boolean invert) {

								//convert (temp)
								ArrayList<SSSNode> subresultstemp = new ArrayList<SSSNode>();
								subresultstemp.addAll(newnodes);

								log("triggering next step after subquerys  _"+sinvertFlag);

								//not in the above yet				
								dothisafter.run(subresultstemp,sinvertFlag);
							}

						};


						SSSNodesWithCommonProperty.getAllNodesWithProperty(cvalue.pred, cvalue.value,doThisAfter,false);

						
						



					}
				});

				}
				/*
				Scheduler.get().scheduleDeferred(new ScheduledCommand() {    
					@Override
					public void execute() {

						if (cvalue.getQueryForPred()!=null){
							error("SubqueryPredicate not yet supported ");

						}
						if (cvalue.getQueryForValue()!=null){
							error("SubqueryValue not yet supported ");
						}

						HashSet<SSSNode> node = SSSNodesWithCommonProperty.getAllNodesWithProperty(cvalue.pred, cvalue.value,null,false);

						//convert (temp)
						ArrayList<SSSNode> subresultstemp = new ArrayList<SSSNode>();
						subresultstemp.addAll(node);

						//not in the above yet				
						dothisafter.run(subresultstemp,sinvertFlag);
					}});*/



			}



		}

		return results;




	}


	//public static void setDebugPanel(DebugDisplayer debugPanel){
	//	DebugPanel=debugPanel;
	//}

	private static void log(String string) {

		SuperSimpleSemantics.log(string);

		//if theres a debug panel set, we log to it
		//if (DebugPanel!=null){			
		//	DebugPanel.log(string);
		//}		
	}

	private static void error(String string) {
		SuperSimpleSemantics.error(string);

		//if theres a debug panel set, we log to it
		//if (DebugPanel!=null){			
		//	DebugPanel.error(string);
		//}				
	}

	/** Get nodes which have all these properties.
	 * Once it finishes its long, complex, process, it passes the result to runWhenDone
	 * 
	 * TODO: slowly add a option into this function to get a instant result rather then a callback. This will be triggered if runWhenDone is null
	 * and rather then return being void, return will the the result
	 * 
	 * @param runWhenDone 
	 * @param runOnEachNewResult 
	 * @return 
	 * 
	 * @return currently experimental. Probably wrong. Maybe very wrong. In future this will be the currentNodes which match the result and the invert state (we need a combined object for this), or null if runWhenDone has been supplied
	 * 
	 * **/
	public static ArrayList<SSSNode> getInsectionOfQuery(final Query hasToHave, DoSomethingWithNodesRunnable runOnEachNewResult, final DoSomethingWithNodesRunnable runWhenDone,final boolean invert) {

		
		if (hasToHave==null){
			log(" --- intersection data is null ");
			return null;
		}

		//establish if we are using a callback system to get the results. If we are, then InstantMode is set to false.	
		//This is the default, and the most powerful method. It allows dynamic cross-site loading of lists and sublists. 
		//if InstantMode is on, the method will instead just return currently known nodes that match all parameters. No extra loading will take place.
		boolean InstantModetemp = false;		
		if (runWhenDone==null){
			InstantModetemp=true;
		}
		final boolean InstantMode = InstantModetemp;		

		log(" -------------------------------------- ");				
		log("Starting Insection query of - "+hasToHave.getAsString());
		log(" -------------------------------------- ");

		Log.info("processing from query object:");

		//boolean firstresult = true;
		boolean invertFlag = false;
		//processNewComboResult dothisafter = new processNewComboResult(runWhenDone);

		final ArrayList<SSSNode> results = new ArrayList<SSSNode>(); //note; this have nodes potentially both added and then  subtracted from it as the dothisafter created below is run potentially many times.
		final int LeftToLoadTotal = hasToHave.size();

		//first we prepare the callback, so that each new node set get intersected with the results as it arrives.
		final DoSomethingWithNodesRunnable  dothisafterForIntersect = new DoSomethingWithNodesRunnable (){	

			private boolean invertFlag=false;
			int LeftToLoad =LeftToLoadTotal;
			Query SourceQuery = hasToHave;

			@Override
			public void run(ArrayList<SSSNode> newnodes, boolean invert) {
				this.invertFlag = invert;
				//we do a hashset to remove dups
				HashSet<SSSNode> resultNodes = new HashSet<SSSNode>(newnodes);				

				Log.info("new results_invert results ="+invertFlag);

				log("current query = "+SourceQuery.getAsString());
				log("invert = "+invertFlag);
				log("current results pre intersect - "+results.toString());
				log("new results to intersect - "+resultNodes.toString());

				if (results.size()==0){

					log("adding nodes as theres no other nodes yet");
					if (invert)	{
						log("This is probably an error; As these nodes should be subtracted notadded!");
						log("We thus delay");

						//delay and run again after!
						final ArrayList<SSSNode> forwardnewnodes = newnodes;
						final DoSomethingWithNodesRunnable forwardthis = this;
						final boolean forwardinvertflag = invertFlag;

						SuperSimpleSemantics.waitFor.scheduleAfter(new Runnable() {    


							@Override
							public void run() {

								log("Triggering delayed action");
								log("invertFlag="+forwardinvertflag);
								log("------------------");

								forwardthis.run(forwardnewnodes, forwardinvertflag);

							}
						});



						//
						return;

					} else {

						results.addAll(resultNodes);


						//firstresult = false;
					}
				} else {

					if (invert)	{
						log("removing nodes:"+resultNodes.size());
						results.removeAll(resultNodes);
					} else {
						log("retaining nodes:"+resultNodes.size());
						results.retainAll(resultNodes);
					}

				}

				log("results are now; - "+results.toString());

				LeftToLoad--;

				log("current left to process intersect = "+LeftToLoad);


				if (LeftToLoad==0){

					SourceQuery.setResultData(results.toString());

					log("triggering next step invert="+SourceQuery.invertResults);

					//run the next bit (which probably is the outer element)
					runWhenDone.run(results,SourceQuery.invertResults);
				}
			}

		};



		//loop to get nodes
		for (QueryElement queryelement : hasToHave) {

			HashSet<SSSNode> node;

			//if the element is a query we regress
			if (queryelement.query!=null){

				Log.info("processing from  a sub query object:");
				Log.info("sub query =:"+queryelement.query.getAsString());
				//invertFlag = queryelement.query.invertResults;
				//ArrayList<SSSNode> subresults =  

				final QueryElement forwardquery = queryelement;

				SuperSimpleSemantics.waitFor.scheduleAfter(new Runnable() {

					@Override
					public void run() {
						processQuery(forwardquery.query,forwardquery.query.invertResults,null,dothisafterForIntersect);

					}
				});


				//Log.info("invert results ="+invertFlag);

			} else {

				final QueryElement forwardquery = queryelement;		
				final Boolean forwardinvertFlag = invertFlag;

				QuerySSSProperty cvalue = forwardquery.prop;

				//if there's no predicate or values needed we run it straight away
				if ((cvalue.getQueryForPred()==null)&&(cvalue.getQueryForValue()==null)){


					//create runnable to process current query element
					Runnable scheduledNodesWithPropertyQuery = setUpPropertyQueryScheduledCommand(
							dothisafterForIntersect, forwardquery,
							forwardinvertFlag,null,null); //Instant mode determines if the runable runs everything instantly, or will wait for additional fileloading.

					if (!InstantMode){
						//set it to run via the waitFor, which might allow interface updates or other things to do while
						//this task is running
						SuperSimpleSemantics.waitFor.scheduleAfter(scheduledNodesWithPropertyQuery);

					} else {

						//if we are not using callbacks run it instantly.
						scheduledNodesWithPropertyQuery.run();

					}


					continue;

				}

				//else we have to tell them to load and wait :(
				//needing both pred and value not yet supported

				if ((cvalue.getQueryForPred()!=null)&&(cvalue.getQueryForValue()!=null)){

					error("SubqueryPredicate&Value support expiremental...");
					final QuerySSSProperty forwardqueryForDualWait = cvalue;

					//trigger a callback to wait for the data before setting up the differed command
					DoSomethingWithNodesRunnable waitForPredicateSubquery1 = new DoSomethingWithNodesRunnable(){
						ArrayList<SSSNode> PredicateNodes ;
						@Override
						public void run(ArrayList<SSSNode> newnodes,
								boolean invert) {

							PredicateNodes = newnodes;
							error("SubqueryPredicate recieved 1");

							//trigger a callback to wait for the data before setting up the differed command
							DoSomethingWithNodesRunnable waitForValueSubquery2 = new DoSomethingWithNodesRunnable(){

								@Override
								public void run(ArrayList<SSSNode> newnodes,
										boolean invert) {

									error("SubqueryValue recived 2: support expiremental...");


									Runnable scheduledNodesWithPropertyQuery = setUpPropertyQueryScheduledCommand(
											dothisafterForIntersect, forwardquery,
											forwardinvertFlag,PredicateNodes,newnodes);

									if (!InstantMode){
										SuperSimpleSemantics.waitFor.scheduleAfter(scheduledNodesWithPropertyQuery);
									} else {
										scheduledNodesWithPropertyQuery.run();
									}

								}

							};

							processQuery(forwardqueryForDualWait.getQueryForValue(),forwardqueryForDualWait.getQueryForValue().invertResults,null,waitForValueSubquery2);


						}

					};

					processQuery(forwardqueryForDualWait.getQueryForPred(),forwardqueryForDualWait.getQueryForPred().invertResults,null,waitForPredicateSubquery1);
					continue;
				}

				if (cvalue.getQueryForPred()!=null){
					error("SubqueryPredicate support expiremental...");
					//trigger a callback to wait for the data before setting up the differed command
					DoSomethingWithNodesRunnable waitForPredicateSubquery = new DoSomethingWithNodesRunnable(){

						@Override
						public void run(ArrayList<SSSNode> newnodes,
								boolean invert) {

							error("SubqueryPredicate support expiremental");

							Runnable scheduledNodesWithPropertyQuery = setUpPropertyQueryScheduledCommand(
									dothisafterForIntersect, forwardquery,
									forwardinvertFlag,newnodes,null);
							if (!InstantMode){
								SuperSimpleSemantics.waitFor.scheduleAfter(scheduledNodesWithPropertyQuery);
							} else {
								scheduledNodesWithPropertyQuery.run();
							}
						}

					};

					processQuery(cvalue.getQueryForPred(),cvalue.getQueryForPred().invertResults,null,waitForPredicateSubquery);
					continue;
				}
				if (cvalue.getQueryForValue()!=null){
					error("SubqueryValue expiremental supported...");
					//trigger a callback to wait for the data before setting up the differed command
					DoSomethingWithNodesRunnable waitForValueSubquery = new DoSomethingWithNodesRunnable(){

						@Override
						public void run(ArrayList<SSSNode> newnodes,
								boolean invert) {

							error("SubqueryValue support expiremental...");



							Runnable scheduledNodesWithPropertyQuery = setUpPropertyQueryScheduledCommand(
									dothisafterForIntersect, forwardquery,
									forwardinvertFlag,null,newnodes);

							if (!InstantMode){
								SuperSimpleSemantics.waitFor.scheduleAfter(scheduledNodesWithPropertyQuery);
							} else {
								scheduledNodesWithPropertyQuery.run();
							}

						}

					};

					processQuery(cvalue.getQueryForValue(),cvalue.getQueryForValue().invertResults,null,waitForValueSubquery);
					continue;

				}



			}


		}

		//runWhenDone.run(results,invertFlag);
		return results;
	}


	//This is a GWT thing that needs to be changed to general Java.
	//The basic idea is to package things up into separately runnable code
	//To ensure things can run asycn and thus allow results to be fetched from serves in the meantime.
	//Or, for that mater, for basic GUI stuff to be updated.
	//
	//How to change this to non-gwt code;
	//We could make this a "Runnable" rather then a Scheduled command.
	//Then we use .run() instead as a temp measure.
	//That would still run straight away though.
	//In order to simulate the ScheduleDefered (which is currently used above)
	//We will need to allow a implementation-specific "wait for GUI" style runnable
	//Which in turn triggers THIS runnable.
	//ho-hum.
	/*
	private static ScheduledCommand setUpPropertyQueryScheduledCommand(
			final DoSomethingWithNodesRunnable dothisafterForIntersect,
			final QueryElement forwardquery, final Boolean forwardinvertFlag, final ArrayList<SSSNode> predicateQueryNodes, final ArrayList<SSSNode> valueQueryNodes) {
		//set up the ScheduledCommand				
		ScheduledCommand scheduledNodesWithPropertyQuery = new ScheduledCommand() {


			//the following is used only if the query has a subquery
			public ArrayList<SSSNode> predicateNodes = predicateQueryNodes;
			public ArrayList<SSSNode> valueNodes = valueQueryNodes;


			@Override
			public void execute() {

				QuerySSSProperty cvalue = forwardquery.prop;

				if ((cvalue.getQueryForPred()==null)&&(cvalue.getQueryForValue()==null)){
					//if no subquerys are specified we trigger a normal search
					SSSNodesWithCommonProperty.getAllNodesWithProperty(cvalue.pred, cvalue.value,dothisafterForIntersect,forwardinvertFlag);
					return;
				}

				// else we check if which subquerys are present
				//if both
				if ((cvalue.getQueryForPred()!=null)&&(cvalue.getQueryForValue()!=null)){

					error("SubqueryPredicate & SubqueryValue is experimental ");

					SSSNodesWithCommonProperty.getAllNodesWithPropertys(predicateNodes, valueNodes,dothisafterForIntersect,forwardinvertFlag);

					return;
				}
				//if predicate
				if (cvalue.getQueryForPred()!=null){
					error("SubqueryPredicate support experimental ");

					if (predicateNodes!=null){

						error("Predicate nodes ready "+predicateNodes.toString());
						ArrayList<SSSNode> valueArray=new ArrayList<SSSNode>();			
						valueArray.add(cvalue.value);

						SSSNodesWithCommonProperty.getAllNodesWithPropertys(predicateNodes, valueArray,dothisafterForIntersect,forwardinvertFlag);
						return;
					}
					//
				}

				if (cvalue.getQueryForValue()!=null){

					if (valueNodes!=null){
						error("Value nodes ready! "+valueNodes.toString());

						ArrayList<SSSNode> predArray=new ArrayList<SSSNode>();			
						predArray.add(cvalue.pred);

						SSSNodesWithCommonProperty.getAllNodesWithPropertys(predArray, valueNodes,dothisafterForIntersect,forwardinvertFlag);
						return;
					} else {
						//error
					}
				}



			}};
			return scheduledNodesWithPropertyQuery;
	}
	 */


	/**
	 * gets all the nodes currently known about which meets the following specifications<br>
	 * Essentially this is a non-asycn version of setUpPropertyQueryScheduledCommand(). <br>
	 * <br>
	 * <br>
	 * @param queryelement - supplies the property being looked for<br>
	 * @param forwardinvertFlag - ?? unsure elsewhere this determines if to invert results when this is part of a larger query. Here its not used yet, but should be given in case its needed later. 
	 * @param predicateQueryNodes - used only if the query has a subquery. Specifies all acceptable predicates. <br>
	 * @param valueQueryNodes     - used only if the query has a subquery. Specifies all acceptable values. <br> 
	 * <br>
	 * @return Nodes which match requirements<br>
	 */
	private static HashSet<SSSNode> getCurrentNodesMatchingThis(
			final QueryElement queryelement, final Boolean forwardinvertFlag, final ArrayList<SSSNode> predicateQueryNodes, final ArrayList<SSSNode> valueQueryNodes) {


		SuperSimpleSemantics.info("setting up query");

		//	HashSet<SSSNode> results = new HashSet<SSSNode>();
		ArrayList<SSSNode> predicateNodes = predicateQueryNodes;
		ArrayList<SSSNode> valueNodes = valueQueryNodes;


		QuerySSSProperty cvalue = queryelement.prop;

		if ((cvalue.getQueryForPred()==null)&&(cvalue.getQueryForValue()==null)){

			//if no subquerys are specified we trigger a normal search
			//SSSNodesWithCommonProperty.getAllNodesWithProperty(cvalue.pred, cvalue.value,dothisafterForIntersect,forwardinvertFlag);			
			return SSSNodesWithCommonProperty.getAllCurrentNodesWithProperty(cvalue.pred, cvalue.value, forwardinvertFlag);
		}

		// else we check if which subqueries are present
		//if both
		if ((cvalue.getQueryForPred()!=null)&&(cvalue.getQueryForValue()!=null)){

			error("SubqueryPredicate & SubqueryValue is expiremental ");
			//
			//SSSNodesWithCommonProperty.getAllNodesWithPropertys(predicateNodes, valueNodes,dothisafterForIntersect,forwardinvertFlag);

			return SSSNodesWithCommonProperty.getAllCurrentNodesWithPropertys(predicateNodes, valueNodes,forwardinvertFlag);
		}
		//if predicate
		if (cvalue.getQueryForPred()!=null){
			error("SubqueryPredicate support expiremental ");

			if (predicateNodes!=null){

				error("Predicate nodes ready "+predicateNodes.toString());
				ArrayList<SSSNode> valueArray=new ArrayList<SSSNode>();			
				valueArray.add(cvalue.value);

				//SSSNodesWithCommonProperty.getAllNodesWithPropertys(predicateNodes, valueArray,dothisafterForIntersect,forwardinvertFlag);

				return SSSNodesWithCommonProperty.getAllCurrentNodesWithPropertys(predicateNodes, valueArray,forwardinvertFlag);
			}
			//
		}

		if (cvalue.getQueryForValue()!=null){

			if (valueNodes!=null){
				error("Value nodes ready! "+valueNodes.toString());

				ArrayList<SSSNode> predArray=new ArrayList<SSSNode>();			
				predArray.add(cvalue.pred);

				//SSSNodesWithCommonProperty.getAllNodesWithPropertys(predArray, valueNodes,dothisafterForIntersect,forwardinvertFlag);

				return  SSSNodesWithCommonProperty.getAllCurrentNodesWithPropertys(predArray, valueNodes,forwardinvertFlag);
			} else {
				//error
			}
		}




		return null;

	}




	/**
	 * function that returns a runnable which itself returns a result...

	 *	It will look at the supplied predicate and value node requirements, then run either
	 *   SSSNodesWithCommonProperty.getAllNodesWithProperty(..) with either the required property.
	 *   
	 *   The required nodes might be themselves a subquery stored in the forwardquery.prop, in which case the supplied arrays are used
	 *   If neither value or predicate is query then the arrays supplied arnt used (and can be null)
	 *   
	 *   Note: if "INSTANT MODE" is turned on getAllCurrentNodesWithProperty is used instead. This should return known matching nodes instantly to dothisafterForIntersect without asycn behavour.
	 *
	 *
	 **/
	private static Runnable setUpPropertyQueryScheduledCommand(
			final DoSomethingWithNodesRunnable dothisafterForIntersect,
			final QueryElement forwardquery, final Boolean forwardinvertFlag, final ArrayList<SSSNode> predicateQueryNodes, final ArrayList<SSSNode> valueQueryNodes) {


		SuperSimpleSemantics.info("setting up query");

		//set up the ScheduledCommand				
		Runnable scheduledNodesWithPropertyQuery = new Runnable() {

			//the following is used only if the query has a subquery
			public ArrayList<SSSNode> predicateNodes = predicateQueryNodes;
			public ArrayList<SSSNode> valueNodes = valueQueryNodes;


			@Override
			public void run() {

				QuerySSSProperty cvalue = forwardquery.prop;

				if ((cvalue.getQueryForPred()==null)&&(cvalue.getQueryForValue()==null)){

					
						SSSNodesWithCommonProperty.getAllNodesWithProperty(cvalue.pred, cvalue.value,dothisafterForIntersect,forwardinvertFlag);
					

					return;
				}

				// else we check if which subqueries are present
				//if both
				if ((cvalue.getQueryForPred()!=null)&&(cvalue.getQueryForValue()!=null)){

					error("SubqueryPredicate & SubqueryValue is expiremental ");

					
						SSSNodesWithCommonProperty.getAllNodesWithPropertys(predicateNodes, valueNodes,dothisafterForIntersect,forwardinvertFlag);
					
					return;
				}
				//if predicate
				if (cvalue.getQueryForPred()!=null){
					error("SubqueryPredicate support expiremental ");

					if (predicateNodes!=null){

						error("Predicate nodes ready "+predicateNodes.toString());
						ArrayList<SSSNode> valueArray=new ArrayList<SSSNode>();			
						valueArray.add(cvalue.value);
						
						SSSNodesWithCommonProperty.getAllNodesWithPropertys(predicateNodes, valueArray,dothisafterForIntersect,forwardinvertFlag);

						



						return;
					}
					//
				}

				if (cvalue.getQueryForValue()!=null){

					if (valueNodes!=null){
						error("Value nodes ready! "+valueNodes.toString());

						ArrayList<SSSNode> predArray=new ArrayList<SSSNode>();			
						predArray.add(cvalue.pred);

						
							SSSNodesWithCommonProperty.getAllNodesWithPropertys(predArray, valueNodes,dothisafterForIntersect,forwardinvertFlag);
						
						return;
					} else {
						//error
					}
				}

			}};

			return scheduledNodesWithPropertyQuery;
	}

	/** Get nodes which have all these properties  **/
	public static ArrayList<SSSNode> getInsectionOfPropertys(
			ArrayList<SSSProperty> hasToHave) {

		//get intersection
		ArrayList<SSSNode> results = new ArrayList<SSSNode>();
		//loop to get nodes
		for (SSSProperty cvalue : hasToHave) {

			HashSet<SSSNode> node = SSSNodesWithCommonProperty.getAllCurrentNodesWithProperty(cvalue.pred, cvalue.value,false);

			Log.info("getting nodes:"+node.size());


			if (results.size()==0){
				results.addAll(node);
			} else {
				results.retainAll(node);
			}
		}
		return results;
	}


	/**
	 * Cleans the results by removing all not found elements and replacing them with SSSNode.NOTFOUND indicators.<br>
	 * <br>
	 * Stores results in cache, removing older cache result if its over the size limit<br>
	 * <br>
	 * @param realQuery<br>
	 * @param newnodes<br>
	 * @return
	 */
	private static ArrayList<SSSNode> cleanResultAndStoreInCache(
			final Query realQuery, ArrayList<SSSNode> newnodes) {
		ArrayList<SSSNode> polishedresult = new ArrayList<SSSNode>();

		if (newnodes.size()==0){
			//if theres no results we put one NOT FOUND in the list 
			polishedresult.add(SSSNode.NOTFOUND);	
		} else {

			//we loop and only add the results that arnt "not founds"
			for (SSSNode resultNode : newnodes) {
				if (!resultNode.equals(SSSNode.NOTFOUND)){
					polishedresult.add(resultNode);
				}

			}


		}
		Log.info("returning results after cleaning");

		//Store the result in the cache
		QueryCache.put(realQuery, polishedresult);
		//remove oldest element if its too big
		if (QueryCache.size()>QueryCacheLimit){				
			QueryCache.remove(0);

		}
		return polishedresult;
	}

	/**<br>
	 * Clears the query cache of all stored results.<br>
	 * Use this if the nodes change<br>
	 * 
	 */
	//* TODO: Make this fire automatically if nodes within it change in any way?
	public void clearQueryCache(){

		QueryCache.clear();
	}


}
